## A file to make the working data for this paper. Lots of recoding. Some merging.
## creates wrkdat.rda

library(readr)
library(here)
library(dplyr)
library(sf)
library(sp)
library(units)

load(here("Data", "survey2.rda"))

source("nonbimatchingfunctions.R")
## identify whites
library(car)
survey2$white <- car::recode(
  survey2$country_of_ancestry.x,
  " 'France'=1;
 'Canada'=1;
 'Ukraine'=1;
 'Great Britain'=1;
 'United Kingdom'=1;
 'Ireland'=1;
 'Finland'=1;
 'United States'=1;
 'Hungary'=1;
 'Netherlands'=1;
 'Spain'=1;
 'Slovenia'=1;
 'Germany'=1;
 'Switzerland'=1;
 'Norway'=1;
 'Belgium'=1;
 'Croatia'=1;
 'Slovakia (Slovak Republic)'=1;
 'Italy'=1;
 'Lithuania'=1;
 'Portugal'=1;
 'Poland'=1;
 'Greece'=1;
 'Czech Republic'=1;
 'Guernsey'=1;
 'Sweden'=1;
 'Australia'=1;
 'Jersey'=1;
 'Romania'=1;
 'Austria'=1;
 'Russian Federation'=1;
 'Isle of Man'=1;
 'Estonia'=1;
 'Latvia'=1;
 'Denmark'=1;
 'Bosnia and Herzegovina'=1;
 'Macedonia, Rep of'=1;
 'Iceland'=1;
 'Serbia'=1;
 'Belarus'=1;
 'New Zealand'=1;
 'Bulgaria'=1;
 'Vatican City State (Holy See)'=1;
 'French Metropolitan'=1;
 'Malta'=1;
 'Albania'=1;
 'NA'=NA;
else=0"
)

with(survey2, table(white, country_of_ancestry.x, exclude = c()))

with(survey2, table(demographics.race.white, country_of_ancestry.x, exclude = c()))

with(survey2, table(demographics.race.white, white, exclude = c()))

## Decision: limit analysis to "whites" but not make a new dataset.

## create percent visible minority variables from census data and add to dataset
survey2$vm.csd <- (survey2$CSDCOL12 / survey2$CSDCOL01)
survey2$vm.fed <- (survey2$FEDCOL12 / survey2$FEDCOL01)
survey2$vm.prov <- (survey2$PROVCOL12 / survey2$PROVCOL01)
survey2$vm.da <- (survey2$DACOL06 / survey2$DACOL04)
survey2$da.pop <- survey2$DACOL04
survey2$csd.pop <- survey2$CSDCOL01
## survey2$age<-(2014-survey2$year_of_birth)

# thedvs<-c("social.capital",
#          "community.resp",
#          "participation",
#          "like.live",
#          "like.neighbourhood",
#          "happy.ethnic.groups",
#          "ethnic.shop", ## goes with happy.ethnic.groups
#          "worried.safety",
#          "share.race",
#          "melting.pot")

## from ../data/CodeVariables.R
# survey2$income.coded<-car::car::recode(survey2$income.new.x,
#                                      " 'Less than $20,000'=1;
# 'Between $20,000 and $30,000'=2;
# 'Between $30,000 and $40,000'=3;
# 'Between $40,000 and $50,000'=4;
# 'Between $50,000 and $60,000'=5;
# 'Between $60,000 and $70,000'=6;
# 'Between $70,000 and $80,000'=7;
# 'Between $80,000 and $90,000'=8;
# 'Between $90,000 and $100,000'=9;
# 'Between $100,000 and $110,000'=10;
# 'Between $110,000 and $120,000'=11;
# 'More than $120,000'=12",
#                                 as.factor=FALSE,
#                                 as.numeric=TRUE)
with(survey2, table(income.new, income.coded, exclude = c()))

survey2$allperceptionsmissing <- rowSums(survey2[, c(
  "own.community.community.percentage.chinese",
  "own.community.community.percentage.black",
  "own.community.community.percentage.latin",
  "own.community.community.percentage.east.indian",
  "own.community.community.percentage.other.asian",
  "own.community.community.percentage.aboriginal",
  "own.community.community.percentage.white"
)]) == 0

survey2$community.help <- car::recode(survey2$own.community.help,
  " 'strongly-agree'=1;
 'agree'=0.75;
 'neither'=0.5;
 'disagree'=0.25;
 'strongly-disagree'=0",
  as.factor = FALSE,
  as.numeric = TRUE
)
survey2$community.get.along <- car::recode(survey2$own.community.get.along,
  " 'strongly-agree'=0;
  'agree'=0.25;
  'neither'=0.5;
  'disagree'=0.75;
  'strongly-disagree'=1",
  as.factor = FALSE,
  as.numeric = TRUE
)

survey2$community.share.values <- car::recode(survey2$own.community.share.values,
  " 'strongly-agree'=0;
  'agree'=0.25;
  'neither'=0.5;
  'disagree'=0.75;
  'strongly-disagree'=1",
  as.factor = FALSE,
  as.numeric = TRUE
)
survey2$graffiti <- car::recode(survey2$own.community.graffiti,
  " 'very-likely'=1;
 'likely'=0.75;
 'fifty-fity'=0.5;
 'unlikely'=0.25;
 'very-unlikely'=0",
  as.factor = FALSE,
  as.numeric = TRUE
)
survey2$organize.library <- car::recode(survey2$own.community.community.organize,
  " 'very-likely'=1;
 'likely'=0.75;
 'fifty-fity'=0.5;
 'unlikely'=0.25;
 'very-unlikely'=0",
  as.factor = FALSE,
  as.numeric = TRUE
)

survey2$sense.community <- car::recode(
  survey2$randomized.district.census.feel.community,
  " 'FALSE'=0;
 'TRUE'=1"
)
survey2$like.live <- car::recode(survey2$randomized.district.like.dislike.census,
  " 'like-alot'=1;
 'like'=0.66666666666666;
 'dislike'=0.3333333333333;
 'dislike-alot'=0",
  as.numeric = TRUE,
  as.factor = F
)
survey2$like.neighbourhood <- car::recode(survey2$racial.conflict.like.neighbourhood,
  " 'like-alot'=1;
  'like'=0.66666666666666;
  'dislike'=0.3333333333333;
  'dislike-alot'=0",
  as.numeric = TRUE,
  as.factor = F
)
survey2$happy.ethnic.groups <- car::recode(survey2$racial.ethnic.happier.with.other.groups,
  " 'strongly-agree'=0;
  'agree'=0.25;
  'neither'=0.5;
  'disagree'=0.75;
  'strongly-disagree'=1",
  as.factor = FALSE,
  as.numeric = TRUE
)
survey2$melting.pot <- car::recode(survey2$racial.ethnic.melting.pot,
  " 'strongly-agree'=1;
 'agree'=0.75;
 'neither'=0.5;
 'disagree'=0.25;
 'strongly-disagree'=0",
  as.factor = FALSE,
  as.numeric = TRUE
)
survey2$worried.safety <- car::recode(survey2$racial.conflict.safe.neighbourhood,
  " 'worried'=1;
 'somewhat-worried'=0.66666666666;
 'not-very-worried'=0.33333333333;
 'not-at-all-worried'=0",
  as.factor = FALSE,
  as.numeric = TRUE
)

survey2$noworrysafety <- 1 - survey2$worried.safety
survey2$share.race <- car::recode(survey2$racial.conflict.housing.ethnic,
  " 'diversity'=1;
 'not-important'=.5;
 'same'=0",
  as.factor = FALSE,
  as.numeric = TRUE
)
survey2$community.issue <- (survey2$own.community.informalpartic)
survey2$community.meeting <- survey2$own.community.meetingpartic
survey2$vote.national <- survey2$randomized.district.national.election
survey2$vote.provincial <- survey2$randomized.district.provincial.election


survey2$participation <- with(survey2, community.issue + community.meeting + vote.national + vote.provincial)
survey2$social.capital <- with(survey2, community.share.values + community.get.along + community.help)
survey2$community.resp <- with(survey2, organize.library + graffiti)
survey2$partic2 <- with(survey2, community.issue + community.meeting)
survey2$vote <- with(survey2, vote.national + vote.provincial)
survey2$like <- with(survey2, like.live + like.neighbourhood)

survey2$participation01 <- rescale01(survey2$participation)
survey2$social.capital01 <- rescale01(survey2$social.capital)
survey2$community.resp01 <- rescale01(survey2$community.resp)
survey2$partic201 <- rescale01(survey2$partic2)
survey2$vote01 <- rescale01(survey2$vote)
survey2$like01 <- rescale01(survey2$like)

survey2$happy.ethnic.groupsOwn <- with(survey2, ifelse(ethnic.shop == "other members of their same ethnic background", happy.ethnic.groups, NA))
survey2$happy.ethnic.groupsMin <- with(survey2, ifelse(ethnic.shop == "ethnic minorities", happy.ethnic.groups, NA))

with(survey2, table(happy.ethnic.groupsOwn, ethnic.shop, exclude = c()))
with(survey2, table(happy.ethnic.groupsMin, ethnic.shop, exclude = c()))

survey2$vm.community.obj <- survey2$community_estimated_vismin / survey2$community_estimated_population

## Years in House
survey2$x.years <- survey2$address.x.years
survey2$x.years <- ifelse(survey2$x.years > 1000 & !is.na(survey2$x.years),
  2014 - survey2$x.years, survey2$x.years
)

survey2$req.lang <- car::recode(survey2$randomized.district.req.lang.immigration,
  " 'strongly-disagree'=0;
 'disagree'=0.25;
 'neither'=0.5;
 'agree'=0.75;
 'strongly-agree'=1;
                                 else=NA",
  as.factor = FALSE
)

survey2$decrease.immigration <- car::recode(
  survey2$randomized.district.increase.immigration,
  " 'decrease-much'=1; 'decrease-little'=0.75; 'same'=0.5; 'increase-little'=0.25; 'increase-much'=0",
  as.factor = FALSE
)

survey2$educnew <- car::recode(survey2$education.x, " 'no schooling'= 'less than hs ';
 'completed elementary school'= 'less than hs ';
 'some secondary / high school'= 'less than hs '")
## check the car::recode
table(survey2$education.x, useNA = "ifany")
table(survey2$educnew, useNA = "ifany")

with(survey2, table(education.x, educnew, useNA = "ifany"))

survey2$immpol <- with(survey2, decrease.immigration + req.lang)
survey2$immpol01 <- rescale01(survey2$immpol)

survey2$sex <- survey2$sex.x

## File after recodes
survey_recodes <- survey2
save(survey_recodes, file = here("Data", "survey_recodes.rda"))

## File with only valid census data
# survey2_valid_pop <- droplevels(subset(survey2, !is.na(vm.da) & !is.na(csd.pop)))

######################
### Add extra census data
### First check the geo ids on the survey data:


## lots of dauids on the survey data. Check them. Glance at them to make sure
## that they are just old repetitions

## head(survey2_valid_pop[, grep("dauid", names(survey2_valid_pop),
##  ignore.case = TRUE, value = TRUE
## )])
## head(survey2_valid_pop[, grep("csduid", names(survey2_valid_pop),
##  ignore.case = TRUE, value = TRUE
## )])
##
## sapply(survey2_valid_pop[, grep("dauid", names(survey2_valid_pop),
##  ignore.case = TRUE, value = TRUE
## )], class)
## sapply(survey2_valid_pop[, grep("csduid", names(survey2_valid_pop),
##  ignore.case = TRUE, value = TRUE
## )], class)
##
## all.equal(survey2_valid_pop$dauid, survey2_valid_pop$dauid.x.x)
## all.equal(survey2_valid_pop$dauid, survey2_valid_pop$dauid.x.y)
## all.equal(survey2_valid_pop$dauid, survey2_valid_pop$dauid.y.x)
## all.equal(survey2_valid_pop$dauid, survey2_valid_pop$dauid.y.y)
##
## all.equal(as.integer(survey2_valid_pop$csduid), survey2_valid_pop$csduid1)
## all.equal(as.integer(survey2_valid_pop$csduid), as.integer(survey2_valid_pop$csduid.csd))
##
## survey2_valid_pop %>%
##  filter(is.na(csduid) | is.na(csduid1) | is.na(csduid.csd)) %>%
##  select(csduid, csduid1, csduid.csd)

## This last line is for the Makefile
system("touch survey_recodes.done")
